(define start-state 1)

(define (tokenize p)
  (let loop ((state start-state)
             (collect (make-collector))
	     (i 0)
             (l 0)
	     (t '()))
    (cond
     ((equal? (peek-char p) #\tab)
      (error 'tokenize "tab character"))
     ((eof-object? (peek-char p))
      (collector-get collect))
     ((token state (peek-char p)) =>
      (lambda (state)
        (loop state collect i (+ l 1) (cons (read-char p) t))))
     ((accepting? state) =>
      (lambda (emitter)
        (let ((tok (emitter i l (list->string (reverse t)))))
	  (when tok (collect-object! collect tok))
	  (loop start-state collect (+ i l) 0 '()))))
     (else (error 'tokenize "no parsable token")))))
